package com.alnex.es.wm;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.util.List;

/**
 * @Author Alnex
 * @CreateTime 2020-07-20 15:40
 * @Descirption 爬虫链家数据 相关代码
 */
public class LianjiaPageProcessor implements PageProcessor {

	private Site site = Site.me().setRetryTimes(3).setSleepTime(300);

	@Override
	public void process(Page page) {
		Html html = page.getHtml();

		List<String> list = html.css(".content__list--item--title a").links().all();
		page.addTargetRequests(list);

		String title = html.xpath("//div[@class='content clear w1150']/p/text()").toString();
		page.putField("title", title);

		page.putField("rent", html.xpath("//p[@class='content__aside--title']/span/text()").toString());
		page.putField("type", html.xpath("//ul[@class='content__aside__list']/allText()").toString());
		page.putField("info", html.xpath("//div[@class='content__article__info']/allText()").toString());
		page.putField("img", html.xpath("//div[@class='content__article__slide__item']/img").toString());

		if (page.getResultItems().get("title") == null) {
			//skip this page
			page.setSkip(true);

			// 分页连接
			for (int i = 0; i <= 100; i++) {
				page.addTargetRequest("https://sh.lianjia.com/zufang/pg" + 1);
			}
		}
	}

	@Override
	public Site getSite() {
		return site;
	}

	public static void main(String[] args) {
		Spider.create(new LianjiaPageProcessor())
				.addUrl("https://sh.lianjia.com/zufang/")
				.addPipeline(new MyPipeline())
				.thread(1).run();
	}
}
